{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "from time import sleep"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 利用API获取用户ID"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'ok': 1, 'data': {'userInfo': {'id': 3655689037, 'screen_name': '欢迎新用户', 'profile_image_url': 'https://tva1.sinaimg.cn/crop.0.0.179.179.180/d9e5634djw1east9pi6bej2050050dfw.jpg', 'profile_url': 'https://m.weibo.cn/u/3655689037?uid=3655689037&luicode=10000011&lfid=1005053655689037', 'statuses_count': 5077, 'verified': False, 'verified_type': -1, 'close_blue_v': False, 'description': '', 'gender': 'f', 'mbtype': 2, 'urank': 12, 'mbrank': 1, 'follow_me': False, 'following': False, 'followers_count': 1608992, 'follow_count': 0, 'cover_image_phone': 'https://tva1.sinaimg.cn/crop.0.0.640.640.640/549d0121tw1egm1kjly3jj20hs0hsq4f.jpg', 'avatar_hd': 'https://ww1.sinaimg.cn/orj480/d9e5634djw1east9pi6bej2050050dfw.jpg', 'like': False, 'like_me': False, 'toolbar_menus': [{'type': 'profile_follow', 'name': '关注', 'pic': '', 'params': {'uid': 3655689037}}, {'type': 'link', 'name': '聊天', 'pic': 'http://h5.sinaimg.cn/upload/2015/06/12/2/toolbar_icon_discuss_default.png', 'params': {'scheme': 'sinaweibo://messagelist?uid=3655689037&nick=欢迎新用户'}, 'scheme': 'https://passport.weibo.cn/signin/welcome?entry=mweibo&r=https%3A%2F%2Fm.weibo.cn%2Fapi%2Fcontainer%2FgetIndex%3Ftype%3Duid%26value%3D3655689037'}, {'type': 'link', 'name': '文章', 'pic': '', 'params': {'scheme': 'sinaweibo://cardlist?containerid=2303190002_445_3655689037_WEIBO_ARTICLE_LIST_DETAIL&count=20'}, 'scheme': 'https://m.weibo.cn/p/index?containerid=2303190002_445_3655689037_WEIBO_ARTICLE_LIST_DETAIL&count=20&luicode=10000011&lfid=1005053655689037'}]}, 'avatar_guide': [], 'fans_scheme': 'https://m.weibo.cn/p/index?containerid=231051_-_fans_intimacy_-_3655689037&luicode=10000011&lfid=1005053655689037', 'follow_scheme': 'https://m.weibo.cn/p/index?containerid=231051_-_followersrecomm_-_3655689037&luicode=10000011&lfid=1005053655689037', 'tabsInfo': {'selectedTab': 1, 'tabs': [{'title': '主页', 'tab_type': 'profile', 'containerid': '2302833655689037'}, {'title': '微博', 'tab_type': 'weibo', 'containerid': '1076033655689037', 'apipath': '/profile/statuses', 'url': '/index/my'}, {'title': '相册', 'tab_type': 'album', 'containerid': '1078033655689037', 'filter_group_info': {'title': '全部照片(0)', 'icon': 'http://u1.sinaimg.cn/upload/2014/06/10/userinfo_icon_album.png', 'icon_name': '专辑', 'icon_scheme': ''}}]}, 'showAppTips': 1, 'scheme': 'sinaweibo://userinfo?uid=3655689037&luicode=10000011&lfid=1005053655689037&from=1110006030'}}\n"
     ]
    }
   ],
   "source": [
    "def get_user_info(uid):\n",
    "    result=requests.get('https://m.weibo.cn/api/container/getIndex?type=uid&value={}'.format(uid))\n",
    "    json_data=result.json()\n",
    "    print(json_data)\n",
    "get_user_info('3655689037')#1350995007"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'user_id': 1350995007, 'screen_name': '我是娜扎', 'profile_image_url': 'https://tvax2.sinaimg.cn/crop.20.0.1202.1202.180/50868c3fly8fhzdg7xwnfj20yi0xejtv.jpg', 'followers_count': 15690226, 'follow_count': 516, 'verified': True, 'containerid': '1076031350995007', 'gender': '女', 'verified_reason': '演员，代表作《择天记》'}\n"
     ]
    }
   ],
   "source": [
    "def get_user_info(uid):\n",
    "    result=requests.get('https://m.weibo.cn/api/container/getIndex?type=uid&value={}'.format(uid))\n",
    "    json_data=result.json()\n",
    "    \n",
    "    if json_data['ok']==1:\n",
    "        userInfo={\n",
    "            'user_id':json_data['data']['userInfo']['id'],\n",
    "            'screen_name':json_data['data']['userInfo']['screen_name'],\n",
    "            'profile_image_url':json_data['data']['userInfo']['profile_image_url'],\n",
    "            #'gender':(json_data['data']['userInfo']['gender']=='f'?'女':'男'),\n",
    "            'followers_count':json_data['data']['userInfo']['followers_count'],\n",
    "            'follow_count':json_data['data']['userInfo']['follow_count'],\n",
    "            'verified':json_data['data']['userInfo']['verified'],\n",
    "            #'verified_reason':(verified?json_data['data']['userInfo']['verified_reason']:'')\n",
    "            'containerid': json_data['data']['tabsInfo']['tabs'][1]['containerid']   # 此字段在获取博文中需要\n",
    "        }\n",
    "        if json_data['data']['userInfo']['gender']=='f':\n",
    "            gender='女'\n",
    "        else:\n",
    "            gender='男'\n",
    "        \n",
    "        if userInfo['verified']:\n",
    "            verified_reason=json_data['data']['userInfo']['verified_reason']\n",
    "        else:\n",
    "            verified_reason=''\n",
    "        userInfo['gender']=gender\n",
    "        userInfo['verified_reason']=verified_reason\n",
    "#    print(userInfo)\n",
    "        \n",
    "        \n",
    "        \n",
    "#get_user_info('1350995007')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_all_posts(uid,containerid):\n",
    "    page=0\n",
    "    posts=[]\n",
    "    while page<20:\n",
    "        result=requests.get('https://m.weibo.cn/api/container/getIndex?type=uid&value={}&containerid={}&page={}'\n",
    "                              .format(str(uid), str(containerid), str(page)))\n",
    "        json_data=result.json()\n",
    "        # 当博文获取完毕，退出循环\n",
    "        if not json_data['data']['cards']:\n",
    "            break\n",
    "        #print(json_data['data']['cards'][0]['mblog']['text'])\n",
    "        for x in json_data['data']['cards']:\n",
    "            if x['card_type']==9:               #不是所有的元素都有mblog键,只有card_type==9的有, 不加上判断条件会报错\n",
    "                posts.append(x['mblog']['text'])\n",
    "        #for x in json_data['data']['cards']:\n",
    "        #    posts.append(x['mblog']['text'])\n",
    "        page+=1\n",
    "        sleep(0.5)\n",
    "    return posts\n",
    "posts=get_all_posts(1350995007,1076031350995007)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# html2text\n",
    "html2text API:\n",
    "https://pypi.org/project/html2text/\n",
    "直接使用html2text.html2text()函数就行了\n",
    "jieba.analyse API:\n",
    "https://github.com/fxsjy/jieba/blob/master/README.md\n",
    "### 基于 TF-IDF 算法的关键词抽取\n",
    "import jieba.analyse\n",
    "\n",
    "jieba.analyse.extract_tags(sentence, topK=20, withWeight=False, allowPOS=())\n",
    "sentence 为待提取的文本\n",
    "topK 为返回几个 TF/IDF 权重最大的关键词，默认值为 20\n",
    "withWeight 为是否一并返回关键词权重值，默认值为 False\n",
    "allowPOS 仅包括指定词性的词，默认值为空，即不筛选\n",
    "jieba.analyse.TFIDF(idf_path=None) 新建 TFIDF 实例，idf_path 为 IDF 频率文件\n",
    "代码示例 （关键词提取）\n",
    "\n",
    "https://github.com/fxsjy/jieba/blob/master/test/extract_tags.py\n",
    "\n",
    "关键词提取所使用逆向文件频率（IDF）文本语料库可以切换成自定义语料库的路径\n",
    "\n",
    "用法： jieba.analyse.set_idf_path(file_name) # file_name为自定义语料库的路径\n",
    "自定义语料库示例：https://github.com/fxsjy/jieba/blob/master/extra_dict/idf.txt.big\n",
    "用法示例：https://github.com/fxsjy/jieba/blob/master/test/extract_tags_idfpath.py\n",
    "关键词提取所使用停止词（Stop Words）文本语料库可以切换成自定义语料库的路径\n",
    "\n",
    "用法： jieba.analyse.set_stop_words(file_name) # file_name为自定义语料库的路径\n",
    "自定义语料库示例：https://github.com/fxsjy/jieba/blob/master/extra_dict/stop_words.txt\n",
    "用法示例：https://github.com/fxsjy/jieba/blob/master/test/extract_tags_stop_words.py\n",
    "关键词一并返回关键词权重值示例\n",
    "\n",
    "用法示例：https://github.com/fxsjy/jieba/blob/master/test/extract_tags_with_weight.py\n",
    "### 基于 TextRank 算法的关键词抽取\n",
    "jieba.analyse.textrank(sentence, topK=20, withWeight=False, allowPOS=('ns', 'n', 'vn', 'v')) 直接使用，接口相同，注意默认过滤词性。\n",
    "jieba.analyse.TextRank() 新建自定义 TextRank 实例\n",
    "算法论文： TextRank: Bringing Order into Texts\n",
    "\n",
    "基本思想:\n",
    "将待抽取关键词的文本进行分词\n",
    "以固定窗口大小(默认为5，通过span属性调整)，词之间的共现关系，构建图\n",
    "计算图中节点的PageRank，注意是无向带权图\n",
    "使用示例:\n",
    "见 test/demo.py\n",
    "\n",
    "词性标注\n",
    "jieba.posseg.POSTokenizer(tokenizer=None) 新建自定义分词器，tokenizer 参数可指定内部使用的 jieba.Tokenizer 分词器。jieba.posseg.dt 为默认词性标注分词器。\n",
    "标注句子分词后每个词的词性，采用和 ictclas 兼容的标记法。\n",
    "用法示例\n",
    ">>> import jieba.posseg as pseg\n",
    ">>> words = pseg.cut(\"我爱北京天安门\")\n",
    ">>> for word, flag in words:\n",
    "...    print('%s %s' % (word, flag))\n",
    "...\n",
    "我 r\n",
    "爱 v\n",
    "北京 ns\n",
    "天安门 ns\n",
    "并行分词\n",
    "原理：将目标文本按行分隔后，把各行文本分配到多个 Python 进程并行分词，然后归并结果，从而获得分词速度的可观提升\n",
    "\n",
    "基于 python 自带的 multiprocessing 模块，目前暂不支持 Windows\n",
    "\n",
    "用法：\n",
    "\n",
    "jieba.enable_parallel(4) # 开启并行分词模式，参数为并行进程数\n",
    "jieba.disable_parallel() # 关闭并行分词模式\n",
    "例子：https://github.com/fxsjy/jieba/blob/master/test/parallel/test_file.py\n",
    "\n",
    "实验结果：在 4 核 3.4GHz Linux 机器上，对金庸全集进行精确分词，获得了 1MB/s 的速度，是单进程版的 3.3 倍。\n",
    "\n",
    "注意：并行分词仅支持默认分词器 jieba.dt 和 jieba.posseg.dt。\n",
    "\n",
    "Tokenize：返回词语在原文的起止位置\n",
    "注意，输入参数只接受 unicode\n",
    "默认模式\n",
    "result = jieba.tokenize(u'永和服装饰品有限公司')\n",
    "for tk in result:\n",
    "    print(\"word %s\\t\\t start: %d \\t\\t end:%d\" % (tk[0],tk[1],tk[2]))\n",
    "word 永和                start: 0                end:2\n",
    "word 服装                start: 2                end:4\n",
    "word 饰品                start: 4                end:6\n",
    "word 有限公司            start: 6                end:10\n",
    "\n",
    "搜索模式\n",
    "result = jieba.tokenize(u'永和服装饰品有限公司', mode='search')\n",
    "for tk in result:\n",
    "    print(\"word %s\\t\\t start: %d \\t\\t end:%d\" % (tk[0],tk[1],tk[2]))\n",
    "word 永和                start: 0                end:2\n",
    "word 服装                start: 2                end:4\n",
    "word 饰品                start: 4                end:6\n",
    "word 有限                start: 6                end:8\n",
    "word 公司                start: 8                end:10\n",
    "word 有限公司            start: 6                end:10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "import jieba.analyse\n",
    "from html2text import html2text\n",
    "content=' '.join([html2text(x) for x in posts])\n",
    "result=jieba.analyse.textrank(content, topK=1000, withWeight=True)\n",
    "keyword={}\n",
    "for x in result:\n",
    "    keyword[x[0]]=x[1]\n",
    "#print(keyword)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 构建词云\n",
    "wordcloud API:\n",
    "https://github.com/amueller/word_cloud\n",
    "https://github.com/amueller/word_cloud/blob/master/examples/simple.py\n",
    "wordcloud没有中文只能够自己加载字体此处使用simhei.ttf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "from PIL import Image\n",
    "from wordcloud import WordCloud, STOPWORDS, ImageColorGenerator\n",
    "import numpy as np\n",
    "alice_mask=np.array(Image.open('alice_color.png'))\n",
    "#f=open('stopwords.txt','r')\n",
    "#stopword=\n",
    "#stopwords暂时没有实现\n",
    "wc=WordCloud(font_path='simhei.ttf',background_color=\"white\", max_words=2000, mask=alice_mask, max_font_size=100, random_state=42)\n",
    "wc.generate_from_frequencies(keyword)\n",
    "image_colors = ImageColorGenerator(alice_mask)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'wc' is not defined",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-1-1b92e4ac7dfe>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mmatplotlib\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpyplot\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0mplt\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      2\u001b[0m \u001b[1;31m# 显示图片\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 3\u001b[1;33m \u001b[0mplt\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mimshow\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mwc\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      4\u001b[0m \u001b[0mplt\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mimshow\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mwc\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mrecolor\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcolor_func\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mimage_colors\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      5\u001b[0m \u001b[0mplt\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0maxis\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"off\"\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;31m# 关闭图像坐标系\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mNameError\u001b[0m: name 'wc' is not defined"
     ],
     "output_type": "error"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "# 显示图片\n",
    "plt.imshow(wc)\n",
    "plt.imshow(wc.recolor(color_func=image_colors))\n",
    "plt.axis(\"off\") # 关闭图像坐标系\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    ""
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
